import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import com.mchange.io.FileUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.wltea.analyzer.lucene.IKAnalyzer;

import java.io.*;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;

public class Scrupy extends BreadthCrawler {

    public Scrupy(String crawlPath, boolean autoParse) {
        super(crawlPath, autoParse);
        this.addSeed("http://blog.csdn.net/");
        this.addSeed("https://blog.csdn.net/");
        getConf().setTopN(100);

    }
    public static long size=0;
    public static List<String> list=new ArrayList<String>();
    public static boolean isNumeric(String str){

        for (int i = str.length();--i>=0;){

            if (!Character.isDigit(str.charAt(i))){

                return false;

            }

        }

        return true;

    }

    @SuppressWarnings("deprecation")
    public void visit(Page page, CrawlDatums crawlDatums) {
        Analyzer analyzer=new IKAnalyzer();
//        System.out.println("??"+page.url());

        if (page.matchUrl("http://blog.csdn.net/.*/article/details/[0-9]+(?!#)")
        ||page.matchUrl("https://blog.csdn.net/.*/article/details/[0-9]+(?!#)")) {
            System.out.println(page.url());

//            try {
//                Elements ns = page.select("a");
//                for(Element n:ns){
//                    String url = n.attr("href");
//                    if(url.matches("https://blog.csdn.net/.*/article/details/.*")){
////                        System.out.println(url);
//                        crawlDatums.add(url);
//                    }
//                }
//            }catch (Exception e){
//                e.printStackTrace();
//            }

            String[] t=page.url().split("/");
            String id=t[t.length-1].split("#")[0];
            id=id.split("\\?")[0];
            String title="";
            if(!isNumeric(id)){
                System.out.println("noooooooooooooooot num");
                return;
            }

            try {
                 title += page.select("h1[class=title-article]").first().text();
//                 System.out.println("title:"+title);
            }catch (Exception e){
                e.printStackTrace();
            }
            Elements content = page.select("p");
            for(Element p : content) {
                String c = p.text();
                title+=c;
            }
            String toWrite="";
            TokenStream tokenStream=analyzer.tokenStream("",title);
            //添加引用,用于获取每个关键词
            CharTermAttribute charTermAttribute = tokenStream.addAttribute(CharTermAttribute.class);
            //添加一个偏移量的引用，记录了关键词的开始位置以及结束位置
            OffsetAttribute offsetAttribute = tokenStream.addAttribute(OffsetAttribute.class);
            //将指针调整到列表的头部
            try {
                tokenStream.reset();
                //遍历关键词列表,incrementToken判断是否结束
                while (tokenStream.incrementToken()) {
                    toWrite+=" "+charTermAttribute;
                }
                toWrite=id+toWrite;
                tokenStream.close();
            }catch (Exception e){
                e.printStackTrace();
            }



            try {
                Connection connection=JdbcUtils.getConnection();
                String sql="INSERT INTO `record`(`url_id`, `url`) VALUES ('"+id+"', '"+page.url()+"') ";

                JdbcUtils.insert(connection,sql,null);
            } catch (Exception throwables) {
                throwables.printStackTrace();
            }


//                System.out.println(System.getProperty("user.dir"));

                File filedir=new File(System.getProperty("user.dir")+File.separator+"data");
                if(!filedir.exists()) {
                    filedir.mkdir();}
//                System.out.println("bytes:"+FileUtils.getBytes(filedir));

                File file =new File("input.txt");
                //if file doesnt exists, then create it

                try{
                if(!file.exists()){
                    file.createNewFile();
                }
                    BufferedWriter out = new BufferedWriter(new OutputStreamWriter(
                            new FileOutputStream(file, true)));

                    out.write(toWrite+"\n");
                    out.flush(); // 把缓存区内容压入文件
                    out.close();
                }catch (IOException e){
                    throw new RuntimeException(e);
                }

                long sizee=1622320128l;

                size=file.length();

                double progress = size/(double)sizee;
                System.out.printf("%.8f\n", progress);
                System.out.println(size);

                if(size>sizee){
                    System.exit(0);
                }
        }
    }
    private static long getTotalSizeOfFilesInDir(final File file) {
        if (file.isFile())
            return file.length();
        final File[] children = file.listFiles();
        long total = 0;
        if (children != null)
            for (final File child : children)
                total += getTotalSizeOfFilesInDir(child);
        return total;
    }

    public static void main(String[] args)   {
        try {
            SqlTool.CreateTable();
        }catch (Exception e){
            e.printStackTrace();
        }
        File f = new File("input");


        Scrupy scrupy = new Scrupy("scrupy2", true);

        scrupy.setThreads(100);

        scrupy.addSeed("http://blog.csdn.net/");
        scrupy.addSeed("https://blog.csdn.net/");


        scrupy.addRegex("https://blog.csdn.net/.*/article/details/[0-9]+(?!#)");
        /*start crawl with depth of 4*/
        try {
            scrupy.start(20);

        }catch (Exception e){

        }




    }
}
